package service;


import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Random;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import javax.jdo.PersistenceManager;

import model.Domain;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.appengine.api.datastore.Category;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.urlfetch.HTTPHeader;
import com.google.appengine.api.urlfetch.HTTPMethod;
import com.google.appengine.api.urlfetch.HTTPRequest;
import com.google.appengine.api.urlfetch.HTTPResponse;
import com.google.appengine.api.urlfetch.URLFetchService;
import com.google.appengine.api.urlfetch.URLFetchServiceFactory;

import data.Data;

public class UpdateDomain {
    
    private URLFetchService fetcher = URLFetchServiceFactory.getURLFetchService();
    
    private Domain domain = new Domain();
    
    private TreeMap<String, String> check = new TreeMap<String, String>();
    
    private Future fDomain = null;
    
    private Future fWhois = null;
    
    private Future fIpAddress = null;
    
    private Future fAlexa = null;
    
    private Future fGoogleRank = null;
    
    private Future fAgeDomain = null;
    
    private Future fValidator = null;
    
    private Future fGoogleService = null;
    
    private Future fMarkosweb = null;
    
    private String content = "";
    
    private String alexaContent = "";
    
    private List<String> googleAnalyticSites = new ArrayList<String>();
    
    private List<String> googleAdsenceSites = new ArrayList<String>();
    
    private List<String> siteSameIp = new ArrayList<String>();
    
    public UpdateDomain(Domain domain) {
        this.domain = domain;
        //Load content from urls
        loadUrlDomain();
        
        loadUrlWhois();
        
        loadUrlIpAddress();
        
        loadUrlAlexa();
        
        loadUrlGoogleRank();
        
        loadUrlAgeDomain();

        
        //Get content form url loaded
        fDomainService();
        
        getWhois();
        
        getIpAddress();
        
        fAlexaService();
        
        getGooglePageRank();
        
        getAgeDomain();
        
        PersistenceManager psm = Data.get().getPersistenceManager();
        
        int ran = new Random().nextInt(1000000);
    	if(this.domain.getAlexa()!=0)
    	{
    		ran = this.domain.getAlexa();
    	}
    	javax.jdo.Query query = psm.newQuery(Domain.class);
    	query.setFilter("alexa < "+ran);
    	query.setOrdering("alexa desc");
    	query.setRange(0, 20);

    	List<Domain> domainList=  (List<Domain>)query.execute();
    	ArrayList<String> nearSite = new ArrayList<String> ();
    	if(domainList.size() > 0)
    	{
    		for (int i = 0; i < domainList.size(); i++) {
    			nearSite.add(domainList.get(i).getName());
			}
    	}
    	this.domain.setNearSite(nearSite);
    	this.domain.setDomain(Utils.returnDomain(this.domain.getName()));
    	domain.setStartName(Utils.returnSubDomainTo(this.domain.getName()).substring(0,3));
    	
    	psm.makePersistent(this.domain);
        
    }    

    /************************************* Load URL *************************************/
    private void loadMarkosweb() {
    	try {
            URL urlMarkosweb = new URL("http://www.markosweb.com./www/" + domain.getName());
            fMarkosweb = fetcher.fetchAsync(urlMarkosweb);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run Markosweb");
        }
    }
    
    private void loadUrlDomain() {
        try {
            URL urlDomain = new URL("http://" + domain.getName());
            fDomain = fetcher.fetchAsync(urlDomain);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't run fDomain");
        }
    }
    
    private void loadUrlWhois() {
        try {
            URL urlWhois = new URL("http://www.whoismind.com/whois/"+Utils.returnSubDomainTo(domain.getName())+".html");
            fWhois = fetcher.fetchAsync(urlWhois);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Can't run fWhois");
        }
    }
    
    private void loadUrlIpAddress() {
        try {
            URL urlIpAddress = new URL("http://www.nearby.org.uk/api/dig.php?l=" + domain.getName());
            fIpAddress = fetcher.fetchAsync(urlIpAddress);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fIpAddress", "Can't run fIpAddress");
        }
    }
    
    private void loadUrlAlexa() {
        try {
            URL urlAlexa = new URL("http://www.alexa.com/siteinfo/" + domain.getName());
            fAlexa = fetcher.fetchAsync(urlAlexa);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't run fAlexa");
        }
    }
    
    private void loadUrlGoogleRank() {
        try {
            URL urlGoogleRank = new URL("http://gpageranks.appspot.com/?url=" + domain.getName());
            fGoogleRank = fetcher.fetchAsync(urlGoogleRank);
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fGoogleRank", "Can't run fGoogleRank");
        }
    }
    
    private void loadUrlAgeDomain() {
        try {
            URL urlAgeDomain = new URL("http://www.seologs.com/dns/domain-check.html");
            HTTPRequest request = new HTTPRequest(urlAgeDomain, HTTPMethod.POST);
            request.setHeader(new HTTPHeader("Content-Type", "application/x-www-form-urlencoded"));
            String params = "query=" + domain.getName() + "&output=cdt";
            request.setPayload(params.getBytes());
            fAgeDomain = fetcher.fetchAsync(request);

        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAgeDomain", "Can't run fAgeDomain");
        }
    }
    

    /************************************* Get content *************************************/
    private void fDomainService() {
        try {
        	HTTPResponse response = (HTTPResponse) fDomain.get();
            
            // if redirects are followed, this returns the final URL we are redirected to
            URL finalUrl = response.getFinalUrl();
            
            String charset="utf-8";
            
            this.content = new String(response.getContent(),charset);
    			
            Document doc = Jsoup.parse(this.content);
            
            int index = doc.select("meta").toString().toLowerCase()
			.indexOf("charset=");
			if (index != -1) {
				charset = doc.select("meta").toString().substring(index + 8)
						.replaceAll("[^\\'\\\"0-9a-zA-Z\\-\\_]+[\\w\\W]+", "")
						.replaceAll("[\\'\\\"]+", "");
			}
            
			this.content = new String(response.getContent(),charset);
			
    		getServerInfo(response);
            
            getSiteInfo();
            getGoogleAnalytic();
            getGoogleAdsense();
            getJavascriptLibrary();
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fDomain", "Can't response fDomain");
        }
    }
    
    private void fAlexaService() {
        try {
        	HTTPResponse response = (HTTPResponse) fAlexa.get();
            
            this.alexaContent = new String(response.getContent(),"utf-8");
            
            getAlexaRank();
            getPopularKeyword();
            getCategory();
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fAlexa", "Can't response fAlexa");
        }
    }
    
    private void getPopularKeyword() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            Elements elements = doc.select("#top-keywords-from-search");
            String keyword = "";
            if(elements.size() > 0)
			{
				for (Element element : elements.first().select("u")) {
					keyword+=element.text()+", ";
				}
			}
            domain.setPopularKeyword(new Text(keyword));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getName());
        }
    }
    
    private void getCategory() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            Elements elements = doc.select("#siteSentences + div").select("a");
            String cate = "";
            if(elements.size() > 0)
			{
            	int i=0;
            	for (Element element : elements) {
    				String categoryName = element.text();
    				cate+=categoryName+" | ";
            	}
			}
            domain.setCategory(new Text(cate));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getServerInfo(HTTPResponse response) {
    	try {
            //Get webServer
            String webServer = "";
            String programmingLanguage = "";
            for (int i = 0; i < response.getHeaders().size(); i++) {
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("server")) {
                    webServer = response.getHeaders().get(i).getValue();
                }
                
                if (response.getHeaders().get(i).getName().equalsIgnoreCase("x-powered-by")) {
                    programmingLanguage = response.getHeaders().get(i).getValue();
                }
                //System.out.println(response.getHeaders().get(i).getName() + " - " + response.getHeaders().get(i).getValue());
            }
            
            this.domain.setServer(new Text(webServer));
            this.domain.setLanguage(new Text(programmingLanguage));
            
        } catch (Exception e) {
            this.check.put("server", domain.getDomain() + " - Webserver not found");
        }
    }
    
    private void getSiteInfo() {
        try {
            Document doc = Jsoup.parse(this.content);
            
            String title = "";
            String description = "";
            String keywords  = "";
            String des = "";
            try {
                title = doc.select("title").first().text();
            } catch (Exception e) {
            	e.printStackTrace();
                this.check.put("title", "Can't get info title");
            }

            try {
                keywords = doc.select("meta[name=keywords]").first().attr("content");
            } catch (Exception e) {
                try {
                    keywords = doc.select("meta[http-equiv=keywords]").first().attr("content");
                } catch (Exception e2) {
                    this.check.put("keywords", "Can't get info keywords");
                }        
            }
            
            try {
                description = doc.select("meta[name=description]").first().attr("content");
            } catch (Exception e) {
                try {
                    description = doc.select("meta[http-equiv=description]").first().attr("content");
                } catch (Exception e2) {
                    if(doc.select("body").first().text().length() > 260)
                    {
                        description = doc.select("body").first().text().substring(0,250).replaceAll("[ ]+[\\w\\W]{0,10}+$", "")+" ...";
                    } else {
                        description = doc.select("body").first().text();
                    }
                }
            }
            
            if(description.length()>260)
            {
                description = description.substring(0,250);
            }
            
            this.domain.setTitle(new Text(title));
            this.domain.setKeyword(new Text(keywords));
            this.domain.setDescription(new Text(description));
            
        } catch (Exception e) {
        	e.printStackTrace();
            this.check.put("siteinfo", "Can't connect to domain " + domain.getDomain());
        }
    }
    
    private void getGoogleAnalytic() {
        int begin = content.indexOf("_gaq.push(['_setAccount', '");
        String analyticAccount = "";
        if(begin>0)
        {
            begin+="_gaq.push(['_setAccount', '".length();
            int end =  content.indexOf("'",begin);
            analyticAccount = content.substring(begin, end);
            begin=analyticAccount.lastIndexOf("-")-1;
            
            analyticAccount = analyticAccount.substring(0, begin);
        }
        else
        {
             begin = content.indexOf("_gaq.push([\"_setAccount\", \"");
            if(begin>0)
            {
                begin+="_gaq.push([\"_setAccount\", \"".length();
                int end =  content.indexOf("\"",begin);
                analyticAccount = content.substring(begin, end);
                
            } else {
                begin = content.indexOf("_uacct");
                if(begin > 0)
                {
                    begin+="_uacct".length();
                    analyticAccount = content.substring(begin,begin+30).replaceAll("^[ =\\'\\\"]+", "").replaceAll("[^a-zA-Z0-9\\-\\_]+[\\w\\W]+", "");
                    begin=analyticAccount.lastIndexOf("-");
                    
                    analyticAccount = analyticAccount.substring(0, begin);
                } else {
                    Pattern p = Pattern.compile("UA-[0-9]+");

                    Matcher m = p.matcher(content);
                    while (m.find()) {
                        analyticAccount = m.group();
                    }
                }
            }
        }
        
        this.domain.setAnalytics(analyticAccount);
    }
    
    private void getGoogleAdsense() {
        int begin = content.indexOf("google_ad_client = \"");
        String adsAccount = "";
        if( begin > 0 )
        {
            begin+="google_ad_client = \"', '".length();
            int end =  content.indexOf("\"",begin);
            adsAccount = content.substring(begin, end);
        }
        
        this.domain.setAdsend(adsAccount);
    }
    
    private void getJavascriptLibrary() {
        List<String> jsLib = new ArrayList<String>();
        if(content.indexOf("jquery")>0)
            jsLib.add("jquery");
        
        if(content.indexOf("prototype.js")>0)
            jsLib.add("prototype");
        
        if(content.indexOf("extjs")>0)
            jsLib.add("extjs");
        
        if(content.indexOf("yui")>0)
            jsLib.add("yui");
        
        if(content.indexOf("dojo")>0)
            jsLib.add("dojo");
        
        if(content.indexOf("gwt")>0)
            jsLib.add("gwt");

        if(content.indexOf("easeljs")>0)
            jsLib.add("easeljs");
        
        this.domain.setJsLib( new Text( StringHelper.listToString(jsLib, ", ")));
    }
    
    private void getWhois() {
        try {
            
            HTTPResponse response = (HTTPResponse) fWhois.get();
            Document doc = Jsoup.parse(new String(response.getContent(),"utf-8"));
            
            String whois = doc.select("#whois-p1").html();
            
            Pattern p = Pattern.compile("[^ :]+@[^ ]+");

	        Matcher m = p.matcher(whois.replaceAll("\\<.*?\\>", " "));

	        ArrayList<String> email = new ArrayList<String>();
	        String listEmail = "";
	        while (m.find()) {
	        	int k=0;
	        	for (int i = 0; i < email.size(); i++) {
					if(email.get(i).equals(m.group().replaceAll("[()]+", "")))
					{
						k=1;
					}
				}
	        	if(k==0)
	        	{
	        		email.add(m.group().replaceAll("[()]+", ""));
	        		listEmail += m.group().replaceAll("[()]+", "")+" , ";
	        	}
	        }
            this.domain.setEmail(new Text(listEmail));
            this.domain.setWhois(new Text(whois));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("fWhois", "Error when get value whois");
        }
    }
    
    
    private void getIpAddress() {
        try {
            HTTPResponse response = (HTTPResponse) fIpAddress.get();
            
            this.domain.setIp(new String(response.getContent()));
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("ipAddress", "Can't get ipAddress for this domain " + domain.getDomain());
        }
    }
    
    private void getAlexaRank() {
        try {
            Document doc = Jsoup.parse(this.alexaContent);
            String strAlexaRank = doc.select("#siteStats").select("tbody tr td div").first().text().trim().replace(",","");
            int alexaRank = 0;
            
            try {
                if(strAlexaRank != null && !strAlexaRank.isEmpty() && !strAlexaRank.equals(""))
                {
                    alexaRank = Integer.parseInt(strAlexaRank.replaceAll("[^0-9]+","")) ;
                }
            } catch (Exception e) {
            	e.printStackTrace();
                check.put("alexaGlobeRank", "Can't get position alexa rank of domain " + domain.getDomain());
            }
            
            this.domain.setAlexa(alexaRank);
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("alexaRank", "Can't get position alexa rank of domain " + domain.getDomain());
        }
    }
    
    private void getGooglePageRank() {
        try {
            HTTPResponse response = (HTTPResponse) fGoogleRank.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            String googlePageRank = "0";
            String google = doc.select(".show").select("span").first().select("font").text().replaceAll("[^0-9]", "");
            try {
                if(google!= null && !google.isEmpty())
                {
                    googlePageRank = google;
                }
            } catch (Exception e) {
            	e.printStackTrace();
                check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
            }
            
            this.domain.setGoogle(new Text(googlePageRank));
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("googlePageRank", "Can't get position google page rank of domain " + domain.getDomain());
        }
    }
    
    private void getAgeDomain() {
        try {
            HTTPResponse response = (HTTPResponse) fAgeDomain.get();
            
            Document doc = Jsoup.parse(new String(response.getContent()));
            String age = doc.select("div[class=KonaBody]").select("p").text();
               String[] ageSplit = age.split("(approximately)");

               if(ageSplit.length ==2)
               {
                   this.domain.setAgeDomain(new Text(ageSplit[1].replaceAll("[):]+", "")));
               }
            
        } catch (Exception e) {
        	e.printStackTrace();
            check.put("age", "Can't get age of domain " + domain.getDomain());
        }
    }
}
